# -*- coding: utf-8 -*-
import random
import re
import sys

import grequests
import time
from lxml import etree
from baiduSpider import get_proxy
from wadeTools_encodingDoc import getDocxWorList
from wadeTools_proxys import WadeProxys
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

logger.info("Start print log")
logger.debug("Do something")
logger.warning("Something maybe fail.")
logger.info("Finish")


HEADERS = [
    'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
]

COOKIES = [
    r'BIDUPSID=B11DC34F571F93B83C40F78363D3577E; PSTM=1588473840; BAIDUID=B11DC34F571F93B8FE95481858135E16:FG=1; BD_UPN=12314353; BDUSS=WYtWn5LaEhSTWFlRUVvQ21OUjUyOHA3TFJvclV5eX5EN35HVWZGcS1BTmlvdGxlRVFBQUFBJCQAAAAAAAAAAAEAAAAuJLwU0uTT6tGp9q32rQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGIVsl5iFbJeR; H_WISE_SIDS=146748_143879_145945_146170_145498_147001_146538_145838_131247_144682_137746_144742_144250_146575_140259_127969_146548_146458_146750_147024_146732_146785_131423_146802_144658_142208_146001_146900_107314_146847_146136_139910_146824_140367_144966_145608_139883_146728_147204_146046_145397_143856_139914_110085; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; ispeed_lsm=2; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03413289766; delPer=0; BD_CK_SAM=1; PSINO=7; BD_HOME=1; sugstore=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_PSSID=31619_1451_21113_31069_31660_31782_31321_30823_26350_22159; H_PS_645EC=554dqK8iNHLvBj7yfbW%2F8F0HvIJ6mwiPR3C1yh3NoVXfmKQxJm8nOlAGBxMJJQARLqjS; BDSVRTM=110',
    r'BAIDUID=16189268A7CCFB29696209538CC20C49:FG=1; PSTM=1562828745; BIDUPSID=6BD5CC05C058022DF6CB672F03EE98F5; NOJS=1; BDUSS=jAzblU5NDFPd2I4WWcxbFVFdmNpZ3FWallqNn5JelFXZzJoU0psd2djSFZLaTFlRVFBQUFBJCQAAAAAAAAAAAEAAAAuJLwU0uTT6tGp9q32rQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWdBV7VnQVeS; H_WISE_SIDS=141182_100806_137933_139405_128063_135846_141003_125695_139148_120197_138471_140853_141345_140334_138878_137985_140174_131247_132551_137745_118893_118858_118855_118818_118803_138165_138883_140260_141210_141368_140267_139057_140202_140592_136863_138585_139171_139626_140077_140113_136196_133847_140793_134256_131423_140311_136537_136752_110085_140325_127969_140954_140593_140864_139802_139886_139408_127416_138313_138426_141194_138942_140682_141191_140597_140962; BD_UPN=12314353; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; delPer=0; BD_CK_SAM=1; BD_HOME=1; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03355511966; uc_login_unique=57281200cd279fe37dccf1aaaf53a9bf; uc_recom_mark=cmVjb21tYXJrXzIxMTc5OTM2; BDRCVFR[SHOOb3ODEBt]=mk3SLVN4HKm; PSINO=6; H_PS_PSSID=1455_31117_21110_30840_31186_30903_30823_31085_26350_31196; COOKIE_SESSION=5_0_9_7_14_21_0_3_9_5_10_0_6062261_0_0_0_1585553495_0_1585553495%7C9%230_0_1585553495%7C1; BDRCVFR[feWj1Vr5u3D]=mk3SLVN4HKm; sugstore=0; H_PS_645EC=8c3edffvokmV8%2FS70gaxw3ocsQMKVbZPdigyICwpqEwt6tARKxpYQ2sa3tj0fpRtW91R',
    r'BAIDUID=A55D609BF6D2BFC9D55001A0665B012A:FG=1; BIDUPSID=A55D609BF6D2BFC9D55001A0665B012A; PSTM=1565409473; H_WISE_SIDS=130611_137735_132923_128699_136650_135964_136630_132549_134982_126062_136436_137157_120154_137001_133981_136365_132910_136455_137691_135846_131246_137743_132378_136680_118896_118873_118852_118820_118805_136687_107315_132783_136799_136430_136095_133351_137901_137221_136861_129656_136196_137104_133847_132552_137467_134046_129643_131423_137466_136740_110085_137863_127969_137625_136612_135416_128196_137695_136635_137096_137208_134383_136413_137450_136988; BDUSS=3RpVXhIbktSdGdmbUJjZnRiazJEVEppTjlPcHdUak1JYjFDT3ladH5xamZjZmxkRVFBQUFBJCQAAAAAAAAAAAEAAAB4NA4naGFpc2hlbmc5OQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAN~k0V3f5NFdZ; uc_login_unique=85d11417a5dfe306fc2a2f3b2eb9828d; uc_recom_mark=cmVjb21tYXJrXzI4MDAyOTQ5; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03357167400; BDRCVFR[tFA6N9pQGI3]=mk3SLVN4HKm; H_PS_PSSID=; delPer=0; PSINO=3; BDORZ=FFFB88E999055A3F8A630C64834BD6D0',
]
headers = {
    'User-Agent': random.choice(HEADERS),
    'Cookie': random.choice(COOKIES),
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
    'Referer': 'https://www.baidu.com',
    'X-Forwarded-For':'8.8.8.8',
}

retry_count = 1
def makeproxies():
    proxy = get_proxy().get("proxy")
    proxies = {
        'http': 'http://' + proxy,
        # 'https': 'https://' + proxy,
    }
    print(proxy)
    return proxies


Rqnum = 0
YZnum = 0
start = time.time()


def get_text(html):
    # 这个函数要求必须html是成品，不能是空
        # print(html)
    doc = etree.HTML(html)  # 获取文档树
    main1 = doc.xpath('//em/text()')
    # print(main1)
    # print("".join(main1))
    baidu_FindWord = ''.join(main1)
    logging.debug(msg=baidu_FindWord)
    # 去掉所有的标点符号，进行匹配
    punc = '~`!#$%^&*()_+-=|\';":/.,?><~·！@#￥%……&*（）——+-=“：’；、。，？》《{}'
    baidu_FindWord= re.sub(r"[%s]+" % punc, "", baidu_FindWord)
    return baidu_FindWord


#生产一个请求函数，返回请求内容
def getUrls(dataList):
    # dataList [url,searchWord]

    #这里根据datalist的长度获取对应的代理api
    proxyObj = WadeProxys()
    proxys = proxyObj.getHeiDongIps(
        url=f'http://ip.ipjldl.com/index.php/api/entry?method=proxyServer.hdtiqu_api_url&packid=7&fa=1&groupid=0&fetch_key=&time=1&qty={len(dataList)}&port=1&format=json&ss=5&css=&dt=&pro=&city=&usertype=4')
    # print(proxys)

    logger.info('代理信息为')
    logger.info(len(proxys))
    req_list = [grequests.get(i[0],headers=headers,timeout=3,params={'kw': i[1]},proxies=proxyObj.getproxys_Str(proxys[index]['IP'],proxys[index]['Port']),verify= False) for index,i in enumerate(dataList)]
    responseList = grequests.map(req_list)
    #返回请求后的数据
    # print(len(responseList))
    return responseList



def checkUseHtml(html,searchWordItem,checkStateList):
    if '百度' not in html:
        return False

    if html == None:
        print(f"5次请求失败，正在切换代理ip尝试第{Rqnum}次尝试")
        # time.sleep(3)
        return False
    elif '验证' in html:
        # print(html)
        # time.sleep(3)
        print(f"该ip出现验证码，切换ip第{YZnum}次尝试")
        return False
    else:
        #进入这个函数必须是经过严格把控，直接可以出内容的，各种问题前边过滤（百度已经成功出内容了）
        baidu_FindWord = get_text(html)
        logger.info('网页内容处理后的信息为:')
        logger.info(baidu_FindWord)
        logger.info(searchWordItem)

        # 对搜索词进行处理

        if searchWordItem in baidu_FindWord:
            checkStateList.append([searchWordItem,baidu_FindWord,True])
        else:
            checkStateList.append([searchWordItem, baidu_FindWord, False])
        # 在这里对比搜索词和内容的一致性，如若一致，就打印，若不一致，就打印不一致

        return True



HEADERS = [
    'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
]

COOKIES = [
    r'BIDUPSID=B11DC34F571F93B83C40F78363D3577E; PSTM=1588473840; BAIDUID=B11DC34F571F93B8FE95481858135E16:FG=1; BD_UPN=12314353; BDUSS=WYtWn5LaEhSTWFlRUVvQ21OUjUyOHA3TFJvclV5eX5EN35HVWZGcS1BTmlvdGxlRVFBQUFBJCQAAAAAAAAAAAEAAAAuJLwU0uTT6tGp9q32rQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGIVsl5iFbJeR; H_WISE_SIDS=146748_143879_145945_146170_145498_147001_146538_145838_131247_144682_137746_144742_144250_146575_140259_127969_146548_146458_146750_147024_146732_146785_131423_146802_144658_142208_146001_146900_107314_146847_146136_139910_146824_140367_144966_145608_139883_146728_147204_146046_145397_143856_139914_110085; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; ispeed_lsm=2; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03413289766; delPer=0; BD_CK_SAM=1; PSINO=7; BD_HOME=1; sugstore=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; H_PS_PSSID=31619_1451_21113_31069_31660_31782_31321_30823_26350_22159; H_PS_645EC=554dqK8iNHLvBj7yfbW%2F8F0HvIJ6mwiPR3C1yh3NoVXfmKQxJm8nOlAGBxMJJQARLqjS; BDSVRTM=110',
    r'BAIDUID=16189268A7CCFB29696209538CC20C49:FG=1; PSTM=1562828745; BIDUPSID=6BD5CC05C058022DF6CB672F03EE98F5; NOJS=1; BDUSS=jAzblU5NDFPd2I4WWcxbFVFdmNpZ3FWallqNn5JelFXZzJoU0psd2djSFZLaTFlRVFBQUFBJCQAAAAAAAAAAAEAAAAuJLwU0uTT6tGp9q32rQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWdBV7VnQVeS; H_WISE_SIDS=141182_100806_137933_139405_128063_135846_141003_125695_139148_120197_138471_140853_141345_140334_138878_137985_140174_131247_132551_137745_118893_118858_118855_118818_118803_138165_138883_140260_141210_141368_140267_139057_140202_140592_136863_138585_139171_139626_140077_140113_136196_133847_140793_134256_131423_140311_136537_136752_110085_140325_127969_140954_140593_140864_139802_139886_139408_127416_138313_138426_141194_138942_140682_141191_140597_140962; BD_UPN=12314353; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; delPer=0; BD_CK_SAM=1; BD_HOME=1; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03355511966; uc_login_unique=57281200cd279fe37dccf1aaaf53a9bf; uc_recom_mark=cmVjb21tYXJrXzIxMTc5OTM2; BDRCVFR[SHOOb3ODEBt]=mk3SLVN4HKm; PSINO=6; H_PS_PSSID=1455_31117_21110_30840_31186_30903_30823_31085_26350_31196; COOKIE_SESSION=5_0_9_7_14_21_0_3_9_5_10_0_6062261_0_0_0_1585553495_0_1585553495%7C9%230_0_1585553495%7C1; BDRCVFR[feWj1Vr5u3D]=mk3SLVN4HKm; sugstore=0; H_PS_645EC=8c3edffvokmV8%2FS70gaxw3ocsQMKVbZPdigyICwpqEwt6tARKxpYQ2sa3tj0fpRtW91R',
    r'BAIDUID=A55D609BF6D2BFC9D55001A0665B012A:FG=1; BIDUPSID=A55D609BF6D2BFC9D55001A0665B012A; PSTM=1565409473; H_WISE_SIDS=130611_137735_132923_128699_136650_135964_136630_132549_134982_126062_136436_137157_120154_137001_133981_136365_132910_136455_137691_135846_131246_137743_132378_136680_118896_118873_118852_118820_118805_136687_107315_132783_136799_136430_136095_133351_137901_137221_136861_129656_136196_137104_133847_132552_137467_134046_129643_131423_137466_136740_110085_137863_127969_137625_136612_135416_128196_137695_136635_137096_137208_134383_136413_137450_136988; BDUSS=3RpVXhIbktSdGdmbUJjZnRiazJEVEppTjlPcHdUak1JYjFDT3ladH5xamZjZmxkRVFBQUFBJCQAAAAAAAAAAAEAAAB4NA4naGFpc2hlbmc5OQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAN~k0V3f5NFdZ; uc_login_unique=85d11417a5dfe306fc2a2f3b2eb9828d; uc_recom_mark=cmVjb21tYXJrXzI4MDAyOTQ5; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a03357167400; BDRCVFR[tFA6N9pQGI3]=mk3SLVN4HKm; H_PS_PSSID=; delPer=0; PSINO=3; BDORZ=FFFB88E999055A3F8A630C64834BD6D0',
]




if __name__ == '__main__':
    checkStateList = []  #记录对比状态
    wordList = getDocxWorList(r'[电影岛赏] - 2021-01-12 这才是我2021开年最想看到的电影.docx')
    # wordList = ['这才是我2021开年最想看到的电影', '原创 岛赏小队 电影岛赏 ', '功能介绍 一个关于电影能坚持说真话的地方', '2021-01-12原文', '唯有影梦满银河',
    #             '前段时间刷微博，一抹可爱的粉色推送出现在我眼前，当我点开它的时候，发现它的头像居然是一条萌萌的、毛茸茸的龙！它叫做“许愿神龙”，这是我与“神龙”的第一次见面，虽然只是匆匆一面，但可爱的神龙形象已经我心中留下了深刻的印象',
    #             '作为2021开年上映的合家欢动画电影，欢乐轻松的题材与画风，注定使《许愿神龙》成为近期与家人朋友一起去电影院的首选', '看完《许愿神龙》后，我最直接的感受就是四个字：超出预期',
    #             '本以为会是一部低幼向的动画，没想到对全年龄向的观众都十分友好，全场的成年观众也都看得津津有味，称得上是老少咸宜', '这可能和每个人的心中都有美好的心愿有关吧',
    #             '无论年龄大小，无论层次高低，大家对于美好生活的愿望都是共通的，而自带好运属性的神龙则可以帮他们达成', '步入新年，见龙有喜，看了这部电影，我觉得是为新的一年开端讨了一个好彩头',
    #             '这条“许愿神龙”不仅在电影中十分抢手，在国内院线的点映情况也十分火爆',
    #             '影片刚刚经过两轮点映，第一轮为元旦3天假期，第二轮为1月9日、10日早十点至晚八点，目前影片的点映票房已破千万，猫眼点映开分9.1，获得了看过观众的一致认可，观众们在影院里经久不绝的笑声已经证明了这部电影的价值',
    #             '这么好的影片，桃花岛观影团的线下活动自然不能缺席', '1月9日、1月10日连续两天，我们在北京和上海举办了《许愿神龙》三场超前观影，还专门设计了“女生专场”和“上海特色专场”的主题',
    #             '观影团的观众有序取票、文明观影，在电影院里一起和这条古灵精怪的神龙度过了一个难忘的下午',
    #             '在北京场的映后，我们还请到了两条威“萌”无比的神龙与观众互动，充满活力的小姐姐和萌娃们纷纷和神龙合照、抱抱，希望能够在它身上沾一沾好运的喜气',
    #             '桃花岛观影团的观众还自发地写下了许多珍贵的评论，全力为这部佳作打call', '这部电影用明亮的色彩呈现了一个靓丽的世界，其中美好的情感非常治愈',
    #             '当我们为了自己美好的心愿而努力的样子被别人记住的时候，也是对他人的一种治愈，哪怕只是一个小小的、跟想见的人一起吃一顿饭这样的心愿，也值得被善待',
    #             '电影中为了自己小小愿望不断努力的男主角思齐，是我们所有人的缩影',
    #             '他在外面念书工作，他的妈妈在家为他做着世界上最好喝的汤；他为了童年玩伴放弃了浮华利欲，神龙也为了朋友做出了影响一生的决定……这种生活中平淡却深刻的亲情和友情是最动人的，正是这种“小爱”汇聚成了人间大爱，这是世界上最大的宝石和最高的权力都换不来的东西',
    #             '看完这部电影以后，我相信它能让快节奏生活中的年轻人们感到放松，也能给我们带来生活的信心',
    #             '当映后大银幕上古灵精怪毛茸茸、可爱萌萌的神龙出现在现场的时候，我突然想抱抱它，岛主给了我这个机会，我整个人都被那个拥抱充满了电量',
    #             '粉红色顺滑的皮毛，太好撸了！而且它在电影里不但是个大吃货，还是个表面傲娇、心里柔软的大可爱，怪不得影厅里的小姐姐们都想人手拥有一条神龙呢！',
    #             '《许愿神龙》中的男主角思齐和我们大多数人一样，努力工作，平平凡凡，善良真诚', '他在电影中被神龙治愈，而我们在匆匆忙忙的人生轨迹中，也需要停下脚步来歇一歇，需要这样的电影来慰藉我们',
    #             '它的剧情虽然简单，但是不做作、不煽情、不说教，踏踏实实地讲好了一个故事，真诚动人',
    #             '有桃花岛在上海的影迷说，《许愿神龙》影片中的上海元素运用的特别契合，尤其是上海大妈的形象，挑剔但又不失可爱，又有着一手好厨艺',
    #             '而影片里的上海老市民的邻里关系和家庭生活状态都刻画得恰到好处，上海的馄饨汤包、弄堂里充满烟火气的生活、浦东和浦西的发展变化、送外卖、卖红薯、广场舞等各种元素都十分接地气',
    #             '神龙的形态也是变化莫测——儿童画、风筝、舞龙舞狮等等，还有很多好玩的梗，是只有看了电影才能get到的，这是创作团队国际化视野和质感的体现', '同时影片云集了众多精英，为这部电影带来了更多惊喜',
    #             '比如我们很熟悉的成龙大哥，我很小的时候，就看了很多成龙大哥的动作电影，包括动画片《成龙历险记》，我相信那是很多人的童年回忆',
    #             '而在这部《许愿神龙》中，我惊喜地发现了它对成龙动作电影的致敬，那些经典的武打动作和场面被高度还原，再加上成龙大哥精彩的配音，让喜爱成龙电影的人找到了某些随着时间流逝而被慢慢遗忘的电影情怀，牛骏峰和薇薇的配音也是为影片增添了更多的青春活力',
    #             '他们与创作者一道，为全世界宣传了中国文化，也递出了极具特色的中国名片', '《许愿神龙》电影中最珍贵的东西，也是我们现实中最珍贵的东西——情感',
    #             '普世情感和人间大爱是一个永恒命题，我们的生活和电影都一直在对它进行解读，而更重要的是珍惜和拥抱它', '从现在开始，拥抱每一份真挚的感情，拥抱每一部走心的电影',
    #             '2020年已经过去，就对着神龙，许下一个愿望，抱一抱它，让阴霾一扫而光吧', '电影岛赏读者群正式上线\n他是传奇最好的代名词\n用8年的时间，看懂一部电影', '还可以在这些平台找到我',
    #             '因为你的分享、点赞、在看', '我足足的精气神儿！']

    dataList = []
    for wordItem in wordList:
        # time.sleep(3)
        searchWord = f'{wordItem}'
        searchWord = searchWord
        print('searchWord----',searchWord)
        url = f"http://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=baidu&wd={searchWord}&fenlei=256&oq=seo&rsv_pq=f910f0f3000fe857&rsv_t=acc3RUQCtu%2FSTxgtcqyM53NKwxbU6Yjah7Ar4s4JFsZQDXG5Tg0PCrpAozI&rqlang=cn&rsv_dl=tb&rsv_enter=0&rsv_btype=t"
        # url = f"https://www.baidu.com/s?wd={searchWord}&pn=1&oq=百度搜索验证机制&ie=utf-8&fenlei=256&rsv_idx=1&rsv_page=1"
        dataList.append([url,wordItem])

    saveData = dataList.copy()  # 保存原始url
    while len(dataList)>0:
        # 每一轮都是一次新的调用，把存在的去除掉，把失败的继续轮转
        proxyApiNeed_num = len(dataList)
        print(f'本轮需要循环的数据列表数量____________________________{len(dataList)}')
        responseList = getUrls(dataList)  #获取请求结果列表
        successList =[]
        #返回列表
        for i in responseList:
            listKeyNum = responseList.index(i)  #先标记这个内容的索引位置
            #当前的搜索词是:
            print('当前的搜索词是:',dataList[listKeyNum][1])
            if i == None:
                print('说明这个请求没有获取成功')
                # 需要做处理# 需要做处理# 需要做处理
                continue
            # print(i.text)

            '''在这个checkUseHtml函数里边进行搜索词和网页句子对比'''
            state = checkUseHtml(i.text,dataList[listKeyNum][1],checkStateList)  #网页内容检查，是否是不正常网页，正常网页代码就返回true，不通过，就返回false进行下一轮请求

            if state == True: #已经正常处理,记录返回的索引，为了定位成功的url，把url也pop出来
                   #这个就是原来本轮请求的索引值
                # print(f'原来本轮请求的索引值{listKeyNum}--{state}')
                successList.append(listKeyNum)
                successList = sorted(successList,reverse=True)  #从大到小排列，从前边弹出会改变list的索引值，倒着弹出就不会i改变前边的索引
                # print(successList)
                # successList = successList.sort(reverse=True)
            else:continue

        for i in successList:
            dataList.pop(i)  # url 弹出成功url，不在进行请求
            # print(f'弹出后的内容{dataList}')


        # print('进入下一轮循环')

    print('完成比对')

    print(checkStateList)

